package extraction;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.LineNumberReader;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Vector;

import opennlp.tools.util.InvalidFormatException;
import WikiPT.Instance;
import WikiPT.Relations;

import com.google.common.collect.LinkedListMultimap;
import com.google.common.collect.Multimap;

public class GenerateSetsPT {

	static Set<String> goldenStandard = new HashSet<String>();
	static Map<String,Integer> sentences = new HashMap<String, Integer>();	
	static Multimap<String, Instance> instances_per_class = LinkedListMultimap.create();
	
	public static void readGoldenStandard( String file) throws IOException {
		  BufferedReader input = new BufferedReader(new FileReader(file));
		  String aux = null;
		  String sentence = null;	  
		  String e1 = null;
		  String e2 = null;	  
		  String type = null;
		  while ((aux = input.readLine()) != null) {
			  if (aux.startsWith("SENTENCE")) {
				  sentence = aux.split(": ")[1];
				  aux = input.readLine();
				  if (aux.equals("")) aux = input.readLine();
				  if (aux.startsWith("MANUALLY CHECKED : IGNORE")) continue;
				  if (aux.startsWith("MANUALLY CHECKED : FALSE")) continue;
				  if (aux.startsWith("MANUALLY CHECKED : TRUE")) {
					  while (!aux.startsWith("*")) {
						  if (aux.startsWith("ENTITY1")) e1 = aux.split(": ")[1].trim();
						  if (aux.startsWith("ENTITY2")) e2 = aux.split(": ")[1].trim();
						  if (aux.startsWith("REL TYPE")) type = aux.split(": ")[1];
						  aux = input.readLine();
					  }
				  StringBuffer i = new StringBuffer(); 			  
				  i.append(sentence);
				  i.append("|");
				  i.append(e1);
				  i.append("|");				
				  i.append(e2);				
				  i.append("|");
				  i.append(type);
				  goldenStandard.add(i.toString());			  
				  }
			  }
		  }
		  input.close();
	  }
	
	public static void generateWikiPT(String[] args, int num_cores, boolean paralell) throws Exception, IOException {
		Relations.initialize();
		PrintWriter outTest = new PrintWriter(new FileWriter("test-data-wikipt.txt"));
		System.out.println("Generating WikiPT test data from: " + args[1] );
		boolean symmetric = Boolean.parseBoolean(args[3]);
		processWikiPT(args[1],outTest,true,symmetric);				
		
		if (paralell) {
			/* Extract training features using several cores */
			LineNumberReader lnr = new LineNumberReader(new FileReader(new File(args[2])));
		    lnr.skip(Long.MAX_VALUE);
		    int num_lines = lnr.getLineNumber();
			int num_instances = num_lines/11;
			int splits_size = num_instances/num_cores;
		    System.out.println("Total lines: " + num_lines);
		    System.out.println("Total instances: " + num_instances);    
		    System.out.println("Number instances per core: " + splits_size + " using " + num_cores + " cores");		    
		    splitFile(args[2],splits_size, num_cores);
		    launchThreads(num_cores,symmetric);
		} else {
			PrintWriter outTrain = new PrintWriter(new FileWriter("train-data-wikipt.txt"));		
			System.out.println("Generating WikiPT train data from: " + args[2]);		
			processWikiPT(args[2],outTrain,false,symmetric);
		}
	}
	
	public static void launchThreads(int num_cores,boolean symmetric) throws InvalidFormatException, FileNotFoundException, IOException, InterruptedException{
	    List<Thread> threads = new ArrayList<Thread>();
	    for (int i = 1; i <= num_cores; i++) {
	      Runnable task = new ExtractFeatures(i,symmetric);
	      Thread worker = new Thread(task);
	      worker.setName(String.valueOf(i));
	      worker.start();
	      threads.add(worker);
	    }
	    /*
	    int running = 0;
	    do {
	      running = 0;
	      for (Thread thread : threads) {
	        if (thread.isAlive()) {
	          running++;
	        }
	      }
	      System.out.println(running + " running threads. ");
	    } while (running > 0);
	    */
	}
		
	public static void splitFile(String file, int splits_size,int num_cores) throws IOException{
		BufferedReader input = new BufferedReader(new FileReader(file));		
		String aux = "not null";
		int instances_processed=0;
		int split_number=1;
		System.out.println("Creating split " + split_number);
		BufferedWriter out = new BufferedWriter(new FileWriter("split_"+split_number+".txt"));
		
		while (aux != null) {
			int count=0;
			while (count<11) {		
				aux = input.readLine();
				out.write(aux+'\n');
				count++;
			}
			instances_processed++;
			if (instances_processed==splits_size) {
				out.close();
				if (split_number!=num_cores) {
					//create new file
					split_number++;
					System.out.println("Creating split " + split_number);
					out = new BufferedWriter(new FileWriter("split_"+split_number+".txt"));
					instances_processed=0;	
				} else break;
			}
		}
		input.close();
	}
		
	public static void processWikiPT(String file, PrintWriter outFile, boolean testData, boolean symmetric) throws Exception {				
		BufferedReader input = new BufferedReader(new FileReader(file));
		String aux = null;
		String sentence = null;
		String type_original = null;
		String e1 = null;
		String e2 = null;
		String type = null;
		boolean checked = false;
		Features f = new Features();
		
		/*
		int instances_processed = 0;
		boolean checked = false;				
		Map<String,Integer> relations_train = new HashMap<String, Integer>();
		Map<String,Integer> relations_test  = new HashMap<String, Integer>();		
		int sentences_train = 0;		
		int words_train = 0;				
		int sentences_test = 0;
		int words_test = 0;		
		ArrayList<Integer> sentences_train_size = new ArrayList<Integer>();
		ArrayList<Integer> sentences_test_size = new ArrayList<Integer>();
		*/
		
		while ((aux = input.readLine()) != null) {
			checked = false;			
			if (aux.startsWith("SENTENCE")) {
				sentence = aux.split(": ")[1];			
				aux = input.readLine(); //skip empty line
				aux = input.readLine();	//read manually checked
				if (aux.startsWith("MANUALLY CHECKED : IGNORE")) continue;				
				if (aux.startsWith("MANUALLY CHECKED : TRUE")) checked = true;
				aux = input.readLine(); //skip empty line
				while (!aux.startsWith("*")) {
					try {
						if (aux.startsWith("ENTITY1")) e1 = aux.split(": ")[1].trim();
						if (aux.startsWith("ENTITY2")) e2 = aux.split(": ")[1].trim();
						if (aux.startsWith("REL TYPE")) type_original = aux.split(": ")[1];
					} catch (Exception e) {
						break;												
					}
					aux = input.readLine();
				}
			}

			if (testData && checked || !testData && (type_original != null && e1 != null && e2 != null)) {
				
				sentence = sentence.replaceAll("&nbsp;", "").replaceAll("&mdash;", "—").replaceAll("&ndash", "–").replaceAll("&bull;", "•");
				sentence = sentence.replaceAll("\\[?URLTOKEN\\s?([A-Za-z0-9íÍÌìàÀáÁâÂâÂãÃçÇéÉêÊóÓõÕôÔúÚüÜ\\.\\s,\\+\\(\\)\\-]+)?\\]?", "");
						
				//eliminar alguns pontos devido ao erro to SentenceDetector
				sentence = sentence.replaceAll(" ca\\. "," ca ").replaceAll(" etc\\. ", " etc ").replaceAll("\\(c\\. ", "(c ").replaceAll(" c\\. ", " c ");				
				sentence = sentence.replaceAll(" Mrs\\. "," Mrs  ").replaceAll("Ph\\.D\\.", "PhD").replaceAll("LL\\.D\\.","LLD").replaceAll("Sc\\.D\\.","ScD").replaceAll(" Mr\\. "," Mr ");				
				sentence = sentence.replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)","$1 $2 $3 $4");
				sentence = sentence.replaceAll("([0-9]+)\\.([0-9]+)\\.([0-9]+)","$1 $2 $3");
				sentence = sentence.replaceAll("([0-9]+)\\.([0-9]+)","$1 $2");
				sentence = sentence.replaceAll(" Lei nº\\. "," Lei nº ").replaceAll(" n°\\. ", " nº ").replaceAll(" nº\\. ", "  nº ").replaceAll("\\(n. ", "(nº ");
				sentence = sentence.replaceAll(" S\\.A\\. "," SA ").replaceAll("Inc\\.","Inc");
				
				if (!testData) {
					String result = Relations.changeRelationType(type_original, e1, e2);
					String[] data = result.split("\\|");
					type = data[0];
					e1 = data[1];
					e2 = data[2];
				} else type=type_original;				
				if (!type.equalsIgnoreCase("ignore")) f.processRelations(sentence,e1,e2,type_original,type,outFile,symmetric);
			}
		}		
		outFile.flush();
		outFile.close();
		input.close();
	}
	

	static void sentenceStatistics(Vector<Integer> sentences_size) {
		int total = 0;
		for (Integer s : sentences_size) { total += s;}
			double average = (double) total / (double) sentences_size.size();   
			System.out.println("Avg. sentence Length: " + average);   
			List<Double> distance_to_average = new Vector<Double>();      
			for (Integer s : sentences_size) { 
	           double difference = (double) s - (average);
	           distance_to_average.add(Math.pow(difference, 2));   
			}
	   double stdvt = 0.0;
	   for (Double d: distance_to_average) {stdvt += d;}   
	   System.out.println("StDev. sentence Length: " +  Math.sqrt(stdvt / (double) distance_to_average.size()));
	}


	static void classStatistics(Map<String, Integer> relations) { int total = 0;
		   for (String c : relations.keySet()) { total += relations.get(c);}
		   double average = (double) total / (double) relations.keySet().size();
		   System.out.println("Avg. class instances: " + average);
		   
		   Vector<Double> distance_to_average = new Vector<Double>();
		   for (String c : relations.keySet()) { 
		           double difference = (double) relations.get(c) - (average);
		           distance_to_average.add(Math.pow(difference, 2));
		   }   
		   double stdvt = 0.0;
		   for (Double d: distance_to_average) {stdvt += d;}   
		   System.out.println("StDev. class instances: " +  Math.sqrt(stdvt / (double) distance_to_average.size()));
	}

	
	static void statsRelations(String type, boolean checked,Map<String, Integer> relations_train, Map<String, Integer> relations_test, int sentences_train, int sentences_test, String sentence, int words_train, int words_test, Vector<Integer> sentences_train_size, Vector<Integer> sentences_test_size) {
		if (checked) {
			sentences_test++;
			sentences_test_size.add(sentence.split("\\s+").length);
			words_test += sentence.split("\\s+").length;
			try {
				int count = relations_test.get(type);
				relations_test.put(type, count++);
			} catch (Exception e) {
				relations_test.put(type, 1);
			}		
			
		} else if (!checked) {
			sentences_train++;
			sentences_train_size.add(sentence.split("\\s+").length);
			words_train += sentence.split("\\s+").length;
			try {
				int count = relations_train.get(type);
				relations_train.put(type, count++);
			} catch (Exception e) {
				relations_train.put(type, 1);
			}	
		}
	}
}